# -*- coding: utf-8 -*-
import scrapy
from spCrawl.items import Frxxz
import json
from scrapy.log import logger

#爬取起点凡人修仙传小说


class FrxxzSpider(scrapy.Spider):
    name = 'Frxxz'
    allowed_domains = ['qidian.com']
    start_urls = ['https://read.qidian.com/chapter/Gyliu2kLjSQ1/BGWJ7ZiYylE1']

    url_pre = "https:"
    count = 0

    def parse(self, response):
        title = response.xpath("//h3[@class='j_chapterName']/text()").extract()
        content = response.xpath("//div[@id='j_chapterBox']/div/div/div[2]").extract()
        # urls = response.xpath("//*[@id='j_readMainWrap']/div[2]/a[3]/@href").extract()
        urls = response.xpath("//*[@id='j_chapterNext']/@href").extract()
        #                      //*[@id="j_chapterBox"]/div[2]/a[3]/@href
        # preUrl = urls[0]
        for var in urls:
            nextUrl = var

        item = Frxxz()
        item["a_title"] = title[0]
        item["c_content"] = content
        # item["preUrl"] = preUrl
        item["b_nextUrl"] = nextUrl
        yield item
        # result
        # if self.count < 10:
        self.count +=1
        next_url = self.url_pre + str(nextUrl)
        logger.info("circle count : %s" % next_url)
        yield scrapy.Request(next_url, callback=self.parse)
